

<!DOCTYPE html>
<html lang="zh-CN" data-default-color-scheme=auto>



<head>
  <meta charset="UTF-8">
  <link rel="apple-touch-icon" sizes="76x76" href="/img/fluid.png">
  <link rel="icon" href="/img/fluid.png">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, shrink-to-fit=no">
  <meta http-equiv="x-ua-compatible" content="ie=edge">
  
  <meta name="theme-color" content="#2f4154">
  <meta name="author" content="John Doe">
  <meta name="keywords" content="">
  
    <meta name="description" content="若发现文章有误，敬请指教，感谢@[TOC]  运行环境  JDK8 Hadoop3.3.0 单节点亦可 Flume1.9 CentOS7  一、参考资料 视频链接Flume官方文档 二、案例介绍&amp;准备知识 Flume官方的架构图： flume本质上就是一个Agent，通过Source（数据源）、Channel（缓冲管道）、Sink（输出源）这么一套工序，将数据从一个地方移动到另一个地方，">
<meta property="og:type" content="article">
<meta property="og:title" content="Flume 笔记2 Flume复制多发案例 _ 配置多个Channel对于相对不同的单个Sink _ 将本地日志文件内容采集到HDFS和另一个本地位置">
<meta property="og:url" content="http://example.com/2022/03/20/Flume%20%E7%AC%94%E8%AE%B02%20Flume%E5%A4%8D%E5%88%B6%E5%A4%9A%E5%8F%91%E6%A1%88%E4%BE%8B%20_%20%E9%85%8D%E7%BD%AE%E5%A4%9A%E4%B8%AAChannel%E5%AF%B9%E4%BA%8E%E7%9B%B8%E5%AF%B9%E4%B8%8D%E5%90%8C%E7%9A%84%E5%8D%95%E4%B8%AASink%20_%20%E5%B0%86%E6%9C%AC%E5%9C%B0%E6%97%A5%E5%BF%97%E6%96%87%E4%BB%B6%E5%86%85%E5%AE%B9%E9%87%87%E9%9B%86%E5%88%B0HDFS%E5%92%8C%E5%8F%A6%E4%B8%80%E4%B8%AA%E6%9C%AC%E5%9C%B0%E4%BD%8D%E7%BD%AE/index.html">
<meta property="og:site_name" content="Hexo">
<meta property="og:description" content="若发现文章有误，敬请指教，感谢@[TOC]  运行环境  JDK8 Hadoop3.3.0 单节点亦可 Flume1.9 CentOS7  一、参考资料 视频链接Flume官方文档 二、案例介绍&amp;准备知识 Flume官方的架构图： flume本质上就是一个Agent，通过Source（数据源）、Channel（缓冲管道）、Sink（输出源）这么一套工序，将数据从一个地方移动到另一个地方，">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://img-blog.csdnimg.cn/6fb7d231ba424dcab685f8448338f64b.png#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/958c02ab8c034fec8bd9ef814c338f85.png#pic_center">
<meta property="og:image" content="https://img-blog.csdnimg.cn/8da241728dc945c5985b724cbd5b41dc.png">
<meta property="og:image" content="https://img-blog.csdnimg.cn/66ea2f394a7a481f83816f6c97bd4a54.png">
<meta property="article:published_time" content="2022-03-20T13:24:56.000Z">
<meta property="article:modified_time" content="2022-08-22T15:43:19.867Z">
<meta property="article:author" content="John Doe">
<meta property="article:tag" content="Flume">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:image" content="https://img-blog.csdnimg.cn/6fb7d231ba424dcab685f8448338f64b.png#pic_center">
  
  
  
  <title>Flume 笔记2 Flume复制多发案例 _ 配置多个Channel对于相对不同的单个Sink _ 将本地日志文件内容采集到HDFS和另一个本地位置 - Hexo</title>

  <link  rel="stylesheet" href="https://lib.baomitu.com/twitter-bootstrap/4.6.1/css/bootstrap.min.css" />



  <link  rel="stylesheet" href="https://lib.baomitu.com/github-markdown-css/4.0.0/github-markdown.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/hint.css/2.7.0/hint.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.css" />



<!-- 主题依赖的图标库，不要自行修改 -->
<!-- Do not modify the link that theme dependent icons -->

<link rel="stylesheet" href="//at.alicdn.com/t/font_1749284_hj8rtnfg7um.css">



<link rel="stylesheet" href="//at.alicdn.com/t/font_1736178_lbnruvf0jn.css">


<link  rel="stylesheet" href="/css/main.css" />


  <link id="highlight-css" rel="stylesheet" href="/css/highlight.css" />
  
    <link id="highlight-css-dark" rel="stylesheet" href="/css/highlight-dark.css" />
  




  <script id="fluid-configs">
    var Fluid = window.Fluid || {};
    Fluid.ctx = Object.assign({}, Fluid.ctx)
    var CONFIG = {"hostname":"example.com","root":"/","version":"1.9.2","typing":{"enable":true,"typeSpeed":70,"cursorChar":"_","loop":false,"scope":[]},"anchorjs":{"enable":true,"element":"h1,h2,h3,h4,h5,h6","placement":"left","visible":"hover","icon":""},"progressbar":{"enable":true,"height_px":3,"color":"#29d","options":{"showSpinner":false,"trickleSpeed":100}},"code_language":{"enable":true,"default":"TEXT"},"copy_btn":true,"image_caption":{"enable":true},"image_zoom":{"enable":true,"img_url_replace":["",""]},"toc":{"enable":true,"placement":"right","headingSelector":"h1,h2,h3,h4,h5,h6","collapseDepth":0},"lazyload":{"enable":true,"loading_img":"/img/loading.gif","onlypost":false,"offset_factor":2},"web_analytics":{"enable":false,"follow_dnt":true,"baidu":null,"google":null,"gtag":null,"tencent":{"sid":null,"cid":null},"woyaola":null,"cnzz":null,"leancloud":{"app_id":null,"app_key":null,"server_url":null,"path":"window.location.pathname","ignore_local":false}},"search_path":"/local-search.xml"};

    if (CONFIG.web_analytics.follow_dnt) {
      var dntVal = navigator.doNotTrack || window.doNotTrack || navigator.msDoNotTrack;
      Fluid.ctx.dnt = dntVal && (dntVal.startsWith('1') || dntVal.startsWith('yes') || dntVal.startsWith('on'));
    }
  </script>
  <script  src="/js/utils.js" ></script>
  <script  src="/js/color-schema.js" ></script>
  


  
<meta name="generator" content="Hexo 6.2.0"></head>


<body>
  

  <header>
    

<div class="header-inner" style="height: 70vh;">
  <nav id="navbar" class="navbar fixed-top  navbar-expand-lg navbar-dark scrolling-navbar">
  <div class="container">
    <a class="navbar-brand" href="/">
      <strong>尤 Ni&#39;s Blog</strong>
    </a>

    <button id="navbar-toggler-btn" class="navbar-toggler" type="button" data-toggle="collapse"
            data-target="#navbarSupportedContent"
            aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
      <div class="animated-icon"><span></span><span></span><span></span></div>
    </button>

    <!-- Collapsible content -->
    <div class="collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav ml-auto text-center">
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/">
                <i class="iconfont icon-home-fill"></i>
                首页
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/archives/">
                <i class="iconfont icon-archive-fill"></i>
                归档
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/categories/">
                <i class="iconfont icon-category-fill"></i>
                分类
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/tags/">
                <i class="iconfont icon-tags-fill"></i>
                标签
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/about/">
                <i class="iconfont icon-user-fill"></i>
                关于
              </a>
            </li>
          
        
        
          <li class="nav-item" id="search-btn">
            <a class="nav-link" target="_self" href="javascript:;" data-toggle="modal" data-target="#modalSearch" aria-label="Search">
              &nbsp;<i class="iconfont icon-search"></i>&nbsp;
            </a>
          </li>
          
        
        
          <li class="nav-item" id="color-toggle-btn">
            <a class="nav-link" target="_self" href="javascript:;" aria-label="Color Toggle">&nbsp;<i
                class="iconfont icon-dark" id="color-toggle-icon"></i>&nbsp;</a>
          </li>
        
      </ul>
    </div>
  </div>
</nav>

  

<div id="banner" class="banner" parallax=true
     style="background: url('/img/default.png') no-repeat center center; background-size: cover;">
  <div class="full-bg-img">
    <div class="mask flex-center" style="background-color: rgba(0, 0, 0, 0.3)">
      <div class="banner-text text-center fade-in-up">
        <div class="h2">
          
            <span id="subtitle" data-typed-text="Flume 笔记2 Flume复制多发案例 _ 配置多个Channel对于相对不同的单个Sink _ 将本地日志文件内容采集到HDFS和另一个本地位置"></span>
          
        </div>

        
          
  <div class="mt-3">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-author" aria-hidden="true"></i>
        John Doe
      </span>
    
    
      <span class="post-meta">
        <i class="iconfont icon-date-fill" aria-hidden="true"></i>
        <time datetime="2022-03-20 21:24" pubdate>
          2022年3月20日 晚上
        </time>
      </span>
    
  </div>

  <div class="mt-1">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-chart"></i>
        
          4.4k 字
        
      </span>
    

    
      <span class="post-meta mr-2">
        <i class="iconfont icon-clock-fill"></i>
        
        
        
          38 分钟
        
      </span>
    

    
    
  </div>


        
      </div>

      
    </div>
  </div>
</div>

</div>

  </header>

  <main>
    
      

<div class="container-fluid nopadding-x">
  <div class="row nomargin-x">
    <div class="side-col d-none d-lg-block col-lg-2">
      

    </div>

    <div class="col-lg-8 nopadding-x-md">
      <div class="container nopadding-x-md" id="board-ctn">
        <div id="board">
          <article class="post-content mx-auto">
            <!-- SEO header -->
            <h1 style="display: none">Flume 笔记2 Flume复制多发案例 _ 配置多个Channel对于相对不同的单个Sink _ 将本地日志文件内容采集到HDFS和另一个本地位置</h1>
            
              <p class="note note-info">
                
                  
                    本文最后更新于：1 小时前
                  
                
              </p>
            
            
              <div class="markdown-body">
                
                <blockquote>
<p>若发现文章有误，敬请指教，感谢<br>@[TOC]</p>
</blockquote>
<h1 id="运行环境"><a href="#运行环境" class="headerlink" title="运行环境"></a>运行环境</h1><hr>
<ul>
<li>JDK8</li>
<li>Hadoop3.3.0 单节点亦可</li>
<li>Flume1.9</li>
<li>CentOS7</li>
</ul>
<h1 id="一、参考资料"><a href="#一、参考资料" class="headerlink" title="一、参考资料"></a>一、参考资料</h1><hr>
<p><a target="_blank" rel="noopener" href="https://www.bilibili.com/video/BV1wf4y1G7EQ?p=20">视频链接</a><br><a target="_blank" rel="noopener" href="https://flume.apache.org/releases/content/1.9.0/FlumeUserGuide.html#avro-source">Flume官方文档</a></p>
<h1 id="二、案例介绍-amp-准备知识"><a href="#二、案例介绍-amp-准备知识" class="headerlink" title="二、案例介绍&amp;准备知识"></a>二、案例介绍&amp;准备知识</h1><hr>
<p><strong>Flume官方的架构图：</strong><br><img src="https://img-blog.csdnimg.cn/6fb7d231ba424dcab685f8448338f64b.png#pic_center" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<p>flume本质上就是一个Agent，通过Source（数据源）、Channel（缓冲管道）、Sink（输出源）这么一套工序，将数据从一个地方移动到另一个地方，那么本次案例则通过这样的思想，实现多个Agent的通信，最终将数据从log日志文件分流到HDFS和另一个本地文件位置。（注：这里暂未涉及拦截器）<br><strong>案例结构图：</strong><br><img src="https://img-blog.csdnimg.cn/958c02ab8c034fec8bd9ef814c338f85.png#pic_center" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"><br>以下内容将用  <code>$FLUME_HOME</code> 代指flume的根目录</p>
<h2 id="2-1-Flume-Memory-Chennel"><a href="#2-1-Flume-Memory-Chennel" class="headerlink" title="2.1 Flume Memory Chennel"></a>2.1 Flume Memory Chennel</h2><p><a target="_blank" rel="noopener" href="https://flume.apache.org/releases/content/1.9.0/FlumeUserGuide.html#avro-source">官方说明</a></p>
<ul>
<li><p>在Flume里什么是Chennel ？</p>
<blockquote>
<p><code>Channels</code> are the repositories where the events are staged on a agent. Source adds the events and Sink removes it.<br>译：<code>Channels</code> 是在Agent存储的事件，Source添加事件，Sink删除事件，言简意赅就是缓冲通道</p>
</blockquote>
</li>
<li><p>在Flume里什么是Memory Chennel ？</p>
<blockquote>
<p>The events are stored in an in-memory queue with configurable max size. It’s ideal for flows that need higher throughput and are prepared to lose the staged data in the event of a agent failures. Required properties are in bold.</p>
<p>译：事件以可配置的最大大小存储在内存队列中。它非常适合需要更高吞吐量的流，并且在代理失败时准备好丢失分段数据。言简意赅就是它是由Ｆｌｕｍｅ内部实现的一种缓冲通道。</p>
</blockquote>
</li>
</ul>
<h2 id="2-2-Flume-Avro-Source"><a href="#2-2-Flume-Avro-Source" class="headerlink" title="2.2 Flume Avro Source"></a>2.2 Flume Avro Source</h2><p><a target="_blank" rel="noopener" href="https://flume.apache.org/releases/content/1.9.0/FlumeUserGuide.html#avro-source">官方说明</a></p>
<h3 id=""><a href="#" class="headerlink" title=""></a></h3><blockquote>
<p>Listens on Avro port and receives events from external Avro client streams. When paired with the built-in Avro Sink on another (previous hop) Flume agent, it can create tiered collection topologies. Required properties are in bold.<br>译：监听Avro端口并从外部Avro客户端流接收事件。当与另一个（上一跳）Flume代理上的内置Avro接收器配对时，它可以创建分层收集拓扑。言简意赅就是avro可通过端口号进行消息通讯，这里是通过avro作为数据源，相当于avro的客户端，同时也可以将avro作为sink，即服务端。avro可实现通过端口号进行通信</p>
</blockquote>
<h1 id="三、准备案例"><a href="#三、准备案例" class="headerlink" title="三、准备案例"></a>三、准备案例</h1><hr>
<h2 id="3-1-配置-Flume"><a href="#3-1-配置-Flume" class="headerlink" title="3.1 配置 Flume"></a>3.1 配置 Flume</h2><p>![在这里插入图片描述](<a target="_blank" rel="noopener" href="https://img-blog.csdnimg.cn/6e88cfd73aa447729fb1e33774ae3307.png#">https://img-blog.csdnimg.cn/6e88cfd73aa447729fb1e33774ae3307.png#</a> &#x3D;60%x)</p>
<p><code>$FLUME_HOME/job/group1/flume-file-flume.conf</code></p>
<figure class="highlight ini"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre></td><td class="code"><pre><code class="hljs ini"><span class="hljs-attr">a1.sources</span> = r1<br><span class="hljs-attr">a1.sinks</span> = k1 k2<br><span class="hljs-attr">a1.channels</span> = c1 c2<br><br><span class="hljs-comment"># 默认启动</span><br><span class="hljs-attr">a1.sources.r1.selector.type</span> = replicating<br><br><span class="hljs-attr">a1.sources.r1.type</span> = exec<br><span class="hljs-attr">a1.sources.r1.command</span> = tail -F /opt/module/flume1.<span class="hljs-number">9</span>/test/test.log<br><span class="hljs-attr">a1.sources.r1.shell</span> = /bin/bash -c<br><br><span class="hljs-attr">a1.sinks.k1.type</span> = avro<br><span class="hljs-attr">a1.sinks.k1.hostname</span> = hadoop101<br><span class="hljs-attr">a1.sinks.k1.port</span> = <span class="hljs-number">4141</span><br><br><span class="hljs-attr">a1.sinks.k2.type</span> = avro<br><span class="hljs-attr">a1.sinks.k2.hostname</span> = hadoop101<br><span class="hljs-attr">a1.sinks.k2.port</span> = <span class="hljs-number">4142</span><br><br><span class="hljs-attr">a1.channels.c1.type</span> = memory<br><span class="hljs-attr">a1.channels.c1.capacity</span> = <span class="hljs-number">1000</span><br><span class="hljs-attr">a1.channels.c1.transactionCapacity</span> = <span class="hljs-number">100</span><br><br><span class="hljs-attr">a1.channels.c2.type</span> = memory<br><span class="hljs-attr">a1.channels.c2.capacity</span> = <span class="hljs-number">1000</span><br><span class="hljs-attr">a1.channels.c2.transactionCapacity</span> = <span class="hljs-number">100</span><br><br><span class="hljs-attr">a1.sources.r1.channels</span> = c1 c2<br><span class="hljs-attr">a1.sinks.k1.channel</span> = c1<br><span class="hljs-attr">a1.sinks.k2.channel</span> = c2<br></code></pre></td></tr></table></figure>

<p><code>flume-flume-hdfs.conf</code></p>
<p>![在这里插入图片描述](<a target="_blank" rel="noopener" href="https://img-blog.csdnimg.cn/f8fdbbedd9344ee4bfba63a19d99491b.png#">https://img-blog.csdnimg.cn/f8fdbbedd9344ee4bfba63a19d99491b.png#</a> &#x3D;50%x)</p>
<figure class="highlight ini"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br></pre></td><td class="code"><pre><code class="hljs ini"><span class="hljs-attr">a2.sources</span> = r1<br><span class="hljs-attr">a2.sinks</span> = k1<br><span class="hljs-attr">a2.channels</span> = c1<br><br><br><span class="hljs-comment"># source端的avro是一个数据接收服务</span><br><span class="hljs-attr">a2.sources.r1.type</span> = avro<br><span class="hljs-attr">a2.sources.r1.bind</span> = hadoop101<br><span class="hljs-attr">a2.sources.r1.port</span> = <span class="hljs-number">4141</span><br><br><span class="hljs-attr">a2.sinks.k1.type</span> = hdfs<br><br><span class="hljs-attr">a2.sinks.k1.hdfs.path</span> = hdfs://hadoop101:<span class="hljs-number">8020</span>/flumeTest/%Y%m%d/%H<br><span class="hljs-comment"># 上传文件的前缀</span><br><span class="hljs-attr">a2.sinks.k1.hdfs.filePrefix</span> = flumeTest-<br><span class="hljs-comment"># 是否按照时间滚动文件夹</span><br><span class="hljs-attr">a2.sinks.k1.hdfs.round</span> = <span class="hljs-literal">true</span><br><span class="hljs-comment"># 多少时间单位创建一个新的文件夹</span><br><span class="hljs-attr">a2.sinks.k1.hdfs.roundValue</span> = <span class="hljs-number">1</span><br><span class="hljs-comment"># 重新定义时间单位</span><br><span class="hljs-attr">a2.sinks.k1.hdfs.roundUnit</span> = hour<br><span class="hljs-comment"># 是否使用本地时间戳</span><br><span class="hljs-attr">a2.sinks.k1.hdfs.useLocalTimeStamp</span> = <span class="hljs-literal">true</span><br><span class="hljs-comment"># 最大支持多少个Event就flush数据</span><br><span class="hljs-attr">a2.sinks.k1.hdfs.batchSize</span>=<span class="hljs-number">100</span><br><span class="hljs-comment"># 设置文件类型，可支持压缩</span><br><span class="hljs-attr">a2.sinks.k1.hdfs.fileType</span> = DataStream<br><span class="hljs-comment"># 多久生成一个新的文件</span><br><span class="hljs-attr">a2.sinks.k1.hdfs.rollInterval</span> = <span class="hljs-number">30</span><br><span class="hljs-comment"># 设置每个文件滚动大小大概为128MB</span><br><span class="hljs-attr">a2.sinks.k1.hdfs.rollSize</span> = <span class="hljs-number">134217700</span><br><br><span class="hljs-comment"># 文件的滚动与Event数量无关</span><br><span class="hljs-attr">a2.sinks.k1.hdfs.rollCount</span> = <span class="hljs-number">0</span><br><br><span class="hljs-attr">a2.channels.c1.type</span> = memory<br><span class="hljs-attr">a2.channels.c1.capacity</span> = <span class="hljs-number">1000</span><br><span class="hljs-attr">a2.channels.c1.transactionCapacity</span> = <span class="hljs-number">100</span><br><br><span class="hljs-attr">a2.sources.r1.channels</span> = c1<br><span class="hljs-attr">a2.sinks.k1.channel</span> = c1<br><br></code></pre></td></tr></table></figure>
<p><code>flume-flume-dir.conf</code><br>![在这里插入图片描述](<a target="_blank" rel="noopener" href="https://img-blog.csdnimg.cn/ebf3c7b39509449eb79a51cc088ecc72.png#">https://img-blog.csdnimg.cn/ebf3c7b39509449eb79a51cc088ecc72.png#</a> &#x3D;50%x)</p>
<figure class="highlight stylus"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><code class="hljs stylus">a3<span class="hljs-selector-class">.sources</span> = r1<br>a3<span class="hljs-selector-class">.sinks</span> = k1<br>a3<span class="hljs-selector-class">.channels</span> = c2<br><br><br>a3<span class="hljs-selector-class">.sources</span><span class="hljs-selector-class">.r1</span><span class="hljs-selector-class">.type</span> = avro<br>a3<span class="hljs-selector-class">.sources</span><span class="hljs-selector-class">.r1</span><span class="hljs-selector-class">.bind</span> = hadoop101<br>a3<span class="hljs-selector-class">.sources</span><span class="hljs-selector-class">.r1</span><span class="hljs-selector-class">.port</span> = <span class="hljs-number">4142</span><br><br>a3<span class="hljs-selector-class">.sinks</span><span class="hljs-selector-class">.k1</span><span class="hljs-selector-class">.type</span> = file_roll<br>a3<span class="hljs-selector-class">.sinks</span><span class="hljs-selector-class">.k1</span><span class="hljs-selector-class">.sink</span><span class="hljs-selector-class">.directory</span> = /opt/module/flume1.<span class="hljs-number">9</span>/testSink<br><br>a3<span class="hljs-selector-class">.channels</span><span class="hljs-selector-class">.c2</span><span class="hljs-selector-class">.type</span> = memory<br>a3<span class="hljs-selector-class">.channels</span><span class="hljs-selector-class">.c2</span><span class="hljs-selector-class">.capacity</span> = <span class="hljs-number">1000</span><br>a3<span class="hljs-selector-class">.channels</span><span class="hljs-selector-class">.c2</span><span class="hljs-selector-class">.transactionCapacity</span> = <span class="hljs-number">100</span><br><br>a3<span class="hljs-selector-class">.sources</span><span class="hljs-selector-class">.r1</span><span class="hljs-selector-class">.channels</span> = c2<br>a3<span class="hljs-selector-class">.sinks</span><span class="hljs-selector-class">.k1</span><span class="hljs-selector-class">.channel</span> = c2<br></code></pre></td></tr></table></figure>
<h2 id="3-2-准备本地sink目录"><a href="#3-2-准备本地sink目录" class="headerlink" title="3.2 准备本地sink目录"></a>3.2 准备本地sink目录</h2><p>由于Flume本地的sink不会自动创建文件夹，所以需提前创建好，否则sink无法输出到指定位置，这里笔者保存的位置在<code>$FLUME_HOME/testSink</code> 可自行修改。最后flume将采集test.log日志的内容（自己随机输入即可）</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><code class="hljs shell"><span class="hljs-meta prompt_">$ </span><span class="language-bash"><span class="hljs-built_in">mkdir</span> /opt/module/flume1.9/testSink</span><br><span class="hljs-meta prompt_"></span><br><span class="hljs-meta prompt_">$ </span><span class="language-bash"><span class="hljs-built_in">touch</span> /opt/module/flume1.9/test/test.log</span><br></code></pre></td></tr></table></figure>
<h1 id="四、测试"><a href="#四、测试" class="headerlink" title="四、测试"></a>四、测试</h1><hr>
<h2 id="4-1-启动-三个-Flume-Agent"><a href="#4-1-启动-三个-Flume-Agent" class="headerlink" title="4.1 启动 三个 Flume Agent"></a>4.1 启动 三个 Flume Agent</h2><p>启动顺序，agent 3 &#x2F; agent 2  ，最后启动agent1：<br>以下命令均在hadoop101节点进行：<br><img src="https://img-blog.csdnimg.cn/8da241728dc945c5985b724cbd5b41dc.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><code class="hljs shell"><span class="hljs-meta prompt_">$ </span><span class="language-bash">bin/flume-ng agent -c conf/ -n a3 -f job/group1/flume-flume-dir.conf &amp;</span><br><span class="hljs-meta prompt_"></span><br><span class="hljs-meta prompt_">$ </span><span class="language-bash">bin/flume-ng agent -c conf/ -n a2 -f job/group1/flume-flume-hdfs.conf &amp;</span><br><span class="hljs-meta prompt_"></span><br><span class="hljs-meta prompt_">$ </span><span class="language-bash">bin/flume-ng agent -c conf/ -n a1 -f job/group1/flume-file-flume.conf &amp;</span><br></code></pre></td></tr></table></figure>

<p>根据配置文件的内容，向本地日志发送消息：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs shell">echo hello &gt;&gt; $FLUME_HOME/test/test.log<br>echo haode &gt;&gt; $FLUME_HOME/test/test.log<br></code></pre></td></tr></table></figure>

<h2 id="4-2-观察测试结果"><a href="#4-2-观察测试结果" class="headerlink" title="4.2 观察测试结果"></a>4.2 观察测试结果</h2><p>可以通过web观察HDFS的情况: <code>http://集群节点主机名:9870/</code>（该地址可通过<code>hdfs-site.xml</code>里的dfs.namenode.http-address属性进行配置，默认为<code>localhost:9870</code>）</p>
<p>出现以下结果，说明Agent1和Agent2都运行成功了<br><img src="https://img-blog.csdnimg.cn/66ea2f394a7a481f83816f6c97bd4a54.png" srcset="/img/loading.gif" lazyload alt="在这里插入图片描述"><br><strong>查看本地文件</strong></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><code class="hljs shell">cd $FLUME_HOME/testSink<br>ls<br>cat 选择后缀为-1的文件<br></code></pre></td></tr></table></figure>
<p>![在这里插入图片描述](<a target="_blank" rel="noopener" href="https://img-blog.csdnimg.cn/368076210e5549dda949c900e2222ebb.png#">https://img-blog.csdnimg.cn/368076210e5549dda949c900e2222ebb.png#</a> &#x3D;50%x)<br>![在这里插入图片描述](<a target="_blank" rel="noopener" href="https://img-blog.csdnimg.cn/3f7cb3ec012340dca52ebd432f494010.png#">https://img-blog.csdnimg.cn/3f7cb3ec012340dca52ebd432f494010.png#</a> &#x3D;50%x)<br>出现这个结果说明Agent1和Agent3都运行成功了，且数据一致。</p>
<p>至此，Flume复制多发的案例就测试完成了。</p>

                
              </div>
            
            <hr/>
            <div>
              <div class="post-metas my-3">
  
    <div class="post-meta mr-3 d-flex align-items-center">
      <i class="iconfont icon-category"></i>
      

<span class="category-chains">
  
  
    
      <span class="category-chain">
        
  <a href="/categories/%E4%B8%AD%E9%97%B4%E4%BB%B6/" class="category-chain-item">中间件</a>
  
  

      </span>
    
  
</span>

    </div>
  
  
    <div class="post-meta">
      <i class="iconfont icon-tags"></i>
      
        <a href="/tags/Flume/">#Flume</a>
      
    </div>
  
</div>


              
  

  <div class="license-box my-3">
    <div class="license-title">
      <div>Flume 笔记2 Flume复制多发案例 _ 配置多个Channel对于相对不同的单个Sink _ 将本地日志文件内容采集到HDFS和另一个本地位置</div>
      <div>http://example.com/2022/03/20/Flume 笔记2 Flume复制多发案例 _ 配置多个Channel对于相对不同的单个Sink _ 将本地日志文件内容采集到HDFS和另一个本地位置/</div>
    </div>
    <div class="license-meta">
      
        <div class="license-meta-item">
          <div>作者</div>
          <div>John Doe</div>
        </div>
      
      
        <div class="license-meta-item license-meta-date">
          <div>发布于</div>
          <div>2022年3月20日</div>
        </div>
      
      
      <div class="license-meta-item">
        <div>许可协议</div>
        <div>
          
            
            
              <a target="_blank" href="https://creativecommons.org/licenses/by/4.0/">
              <span class="hint--top hint--rounded" aria-label="BY - 署名">
                <i class="iconfont icon-by"></i>
              </span>
              </a>
            
          
        </div>
      </div>
    </div>
    <div class="license-icon iconfont"></div>
  </div>



              
                <div class="post-prevnext my-3">
                  <article class="post-prev col-6">
                    
                    
                      <a href="/2022/03/20/Flume%20%E7%AC%94%E8%AE%B03%20source%E5%A4%9A%E8%B7%AF%E5%A4%8D%E7%94%A8%E9%80%89%E6%8B%A9%E5%99%A8%20_%20%E7%AE%80%E5%8D%95%E7%9A%84%E6%8B%A6%E6%88%AA%E5%99%A8%E5%AE%9E%E7%8E%B0%20_%20%E4%B8%8D%E5%90%8C%E9%9B%86%E7%BE%A4%E8%8A%82%E7%82%B9%E7%9A%84%E6%95%B0%E6%8D%AE%E5%88%86%E6%B5%81/" title="Flume 笔记3 source多路复用选择器 _ 简单的拦截器实现 _ 不同集群节点的数据分流">
                        <i class="iconfont icon-arrowleft"></i>
                        <span class="hidden-mobile">Flume 笔记3 source多路复用选择器 _ 简单的拦截器实现 _ 不同集群节点的数据分流</span>
                        <span class="visible-mobile">上一篇</span>
                      </a>
                    
                  </article>
                  <article class="post-next col-6">
                    
                    
                      <a href="/2022/03/19/Flume%20%E7%AC%94%E8%AE%B01%20%E4%BD%BF%E7%94%A8Flume%E6%94%B6%E9%9B%86netcat%E7%AB%AF%E5%8F%A3%E9%80%9A%E4%BF%A1%E7%9A%84%E4%BF%A1%E6%81%AF%E5%88%B0%E6%9C%AC%E5%9C%B0%E6%96%87%E4%BB%B6/" title="Flume 笔记1 使用Flume收集netcat端口通信的信息到本地文件">
                        <span class="hidden-mobile">Flume 笔记1 使用Flume收集netcat端口通信的信息到本地文件</span>
                        <span class="visible-mobile">下一篇</span>
                        <i class="iconfont icon-arrowright"></i>
                      </a>
                    
                  </article>
                </div>
              
            </div>

            
          </article>
        </div>
      </div>
    </div>

    <div class="side-col d-none d-lg-block col-lg-2">
      
  <aside class="sidebar" style="margin-left: -1rem">
    <div id="toc">
  <p class="toc-header"><i class="iconfont icon-list"></i>&nbsp;目录</p>
  <div class="toc-body" id="toc-body"></div>
</div>



  </aside>


    </div>
  </div>
</div>





  



  



  



  



  







    

    
      <a id="scroll-top-button" aria-label="TOP" href="#" role="button">
        <i class="iconfont icon-arrowup" aria-hidden="true"></i>
      </a>
    

    
      <div class="modal fade" id="modalSearch" tabindex="-1" role="dialog" aria-labelledby="ModalLabel"
     aria-hidden="true">
  <div class="modal-dialog modal-dialog-scrollable modal-lg" role="document">
    <div class="modal-content">
      <div class="modal-header text-center">
        <h4 class="modal-title w-100 font-weight-bold">搜索</h4>
        <button type="button" id="local-search-close" class="close" data-dismiss="modal" aria-label="Close">
          <span aria-hidden="true">&times;</span>
        </button>
      </div>
      <div class="modal-body mx-3">
        <div class="md-form mb-5">
          <input type="text" id="local-search-input" class="form-control validate">
          <label data-error="x" data-success="v" for="local-search-input">关键词</label>
        </div>
        <div class="list-group" id="local-search-result"></div>
      </div>
    </div>
  </div>
</div>

    

    
  </main>

  <footer>
    <div class="footer-inner">
  
    <div class="footer-content">
       <a href="https://hexo.io" target="_blank" rel="nofollow noopener"><span>Hexo</span></a> <i class="iconfont icon-love"></i> <a href="https://github.com/fluid-dev/hexo-theme-fluid" target="_blank" rel="nofollow noopener"><span>Fluid</span></a> 
    </div>
  
  
  
  
</div>

  </footer>

  <!-- Scripts -->
  
  <script  src="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.js" ></script>
  <link  rel="stylesheet" href="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.css" />

  <script>
    NProgress.configure({"showSpinner":false,"trickleSpeed":100})
    NProgress.start()
    window.addEventListener('load', function() {
      NProgress.done();
    })
  </script>


<script  src="https://lib.baomitu.com/jquery/3.6.0/jquery.min.js" ></script>
<script  src="https://lib.baomitu.com/twitter-bootstrap/4.6.1/js/bootstrap.min.js" ></script>
<script  src="/js/events.js" ></script>
<script  src="/js/plugins.js" ></script>


  <script  src="https://lib.baomitu.com/typed.js/2.0.12/typed.min.js" ></script>
  <script>
    (function (window, document) {
      var typing = Fluid.plugins.typing;
      var subtitle = document.getElementById('subtitle');
      if (!subtitle || !typing) {
        return;
      }
      var text = subtitle.getAttribute('data-typed-text');
      
        typing(text);
      
    })(window, document);
  </script>




  
    <script  src="/js/img-lazyload.js" ></script>
  




  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/tocbot/4.18.2/tocbot.min.js', function() {
    var toc = jQuery('#toc');
    if (toc.length === 0 || !window.tocbot) { return; }
    var boardCtn = jQuery('#board-ctn');
    var boardTop = boardCtn.offset().top;

    window.tocbot.init({
      tocSelector     : '#toc-body',
      contentSelector : '.markdown-body',
      headingSelector : CONFIG.toc.headingSelector || 'h1,h2,h3,h4,h5,h6',
      linkClass       : 'tocbot-link',
      activeLinkClass : 'tocbot-active-link',
      listClass       : 'tocbot-list',
      isCollapsedClass: 'tocbot-is-collapsed',
      collapsibleClass: 'tocbot-is-collapsible',
      collapseDepth   : CONFIG.toc.collapseDepth || 0,
      scrollSmooth    : true,
      headingsOffset  : -boardTop
    });
    if (toc.find('.toc-list-item').length > 0) {
      toc.css('visibility', 'visible');
    }
  });
</script>


  <script src=https://lib.baomitu.com/clipboard.js/2.0.10/clipboard.min.js></script>

  <script>Fluid.plugins.codeWidget();</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/anchor-js/4.3.1/anchor.min.js', function() {
    window.anchors.options = {
      placement: CONFIG.anchorjs.placement,
      visible  : CONFIG.anchorjs.visible
    };
    if (CONFIG.anchorjs.icon) {
      window.anchors.options.icon = CONFIG.anchorjs.icon;
    }
    var el = (CONFIG.anchorjs.element || 'h1,h2,h3,h4,h5,h6').split(',');
    var res = [];
    for (var item of el) {
      res.push('.markdown-body > ' + item.trim());
    }
    if (CONFIG.anchorjs.placement === 'left') {
      window.anchors.options.class = 'anchorjs-link-left';
    }
    window.anchors.add(res.join(', '));
  });
</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.js', function() {
    Fluid.plugins.fancyBox();
  });
</script>


  <script>Fluid.plugins.imageCaption();</script>

  <script  src="/js/local-search.js" ></script>





<!-- 主题的启动项，将它保持在最底部 -->
<!-- the boot of the theme, keep it at the bottom -->
<script  src="/js/boot.js" ></script>


  

  <noscript>
    <div class="noscript-warning">博客在允许 JavaScript 运行的环境下浏览效果更佳</div>
  </noscript>
</body>
</html>
